{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3、5.4  PyTorchでDQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# パッケージのimport\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 動画の描画関数の宣言\n",
    "# 参考URL http://nbviewer.jupyter.org/github/patrickmineault\n",
    "# /xcorr-notebooks/blob/master/Render%20OpenAI%20gym%20as%20GIF.ipynb\n",
    "from JSAnimation.IPython_display import display_animation\n",
    "from matplotlib import animation\n",
    "from IPython.display import display\n",
    "\n",
    "\n",
    "def display_frames_as_gif(frames):\n",
    "    \"\"\"\n",
    "    Displays a list of frames as a gif, with controls\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(frames[0].shape[1]/72.0, frames[0].shape[0]/72.0),\n",
    "               dpi=72)\n",
    "    patch = plt.imshow(frames[0])\n",
    "    plt.axis('off')\n",
    "\n",
    "    def animate(i):\n",
    "        patch.set_data(frames[i])\n",
    "\n",
    "    anim = animation.FuncAnimation(plt.gcf(), animate, frames=len(frames),\n",
    "                                   interval=50)\n",
    "\n",
    "    anim.save('movie_cartpole_DQN.mp4')  # 動画のファイル名と保存です\n",
    "    display(display_animation(anim, default_mode='loop'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tr(name_a='名前Aです', value_b=100)\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "# 本コードでは、namedtupleを使用します。\n",
    "# namedtupleを使うことで、値をフィールド名とペアで格納できます。\n",
    "# すると値に対して、フィールド名でアクセスできて便利です。\n",
    "# https://docs.python.jp/3/library/collections.html#collections.namedtuple\n",
    "# 以下は使用例です\n",
    "\n",
    "from collections import namedtuple\n",
    "\n",
    "Tr = namedtuple('tr', ('name_a', 'value_b'))\n",
    "Tr_object = Tr('名前Aです', 100)\n",
    "\n",
    "print(Tr_object)  # 出力：tr(name_a='名前Aです', value_b=100)\n",
    "print(Tr_object.value_b)  # 出力：100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# namedtupleを生成\n",
    "from collections import namedtuple\n",
    "\n",
    "Transition = namedtuple(\n",
    "    'Transition', ('state', 'action', 'next_state', 'reward'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定数の設定\n",
    "ENV = 'CartPole-v0'  # 使用する課題名\n",
    "GAMMA = 0.99  # 時間割引率\n",
    "MAX_STEPS = 200  # 1試行のstep数\n",
    "NUM_EPISODES = 500  # 最大試行回数\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 経験を保存するメモリクラスを定義します\n",
    "\n",
    "\n",
    "class ReplayMemory:\n",
    "\n",
    "    def __init__(self, CAPACITY):\n",
    "        self.capacity = CAPACITY  # メモリの最大長さ\n",
    "        self.memory = []  # 経験を保存する変数\n",
    "        self.index = 0  # 保存するindexを示す変数\n",
    "\n",
    "    def push(self, state, action, state_next, reward):\n",
    "        '''transition = (state, action, state_next, reward)をメモリに保存する'''\n",
    "\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)  # メモリが満タンでないときは足す\n",
    "\n",
    "        # namedtupleのTransitionを使用し、値とフィールド名をペアにして保存します\n",
    "        self.memory[self.index] = Transition(state, action, state_next, reward)\n",
    "\n",
    "        self.index = (self.index + 1) % self.capacity  # 保存するindexを1つずらす\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        '''batch_size分だけ、ランダムに保存内容を取り出す'''\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        '''関数lenに対して、現在の変数memoryの長さを返す'''\n",
    "        return len(self.memory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# エージェントが持つ脳となるクラスです、DQNを実行します\n",
    "# Q関数をディープラーニングのネットワークをクラスとして定義\n",
    "\n",
    "import random\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "CAPACITY = 10000\n",
    "\n",
    "\n",
    "class Brain:\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        self.num_actions = num_actions  # CartPoleの行動（右に左に押す）の2を取得\n",
    "\n",
    "        # 経験を記憶するメモリオブジェクトを生成\n",
    "        self.memory = ReplayMemory(CAPACITY)\n",
    "\n",
    "        # ニューラルネットワークを構築\n",
    "        self.model = nn.Sequential()\n",
    "        self.model.add_module('fc1', nn.Linear(num_states, 32))\n",
    "        self.model.add_module('relu1', nn.ReLU())\n",
    "        self.model.add_module('fc2', nn.Linear(32, 32))\n",
    "        self.model.add_module('relu2', nn.ReLU())\n",
    "        self.model.add_module('fc3', nn.Linear(32, num_actions))\n",
    "\n",
    "        print(self.model)  # ネットワークの形を出力\n",
    "\n",
    "        # 最適化手法の設定\n",
    "        self.optimizer = optim.Adam(self.model.parameters(), lr=0.0001)\n",
    "\n",
    "    def replay(self):\n",
    "        '''Experience Replayでネットワークの結合パラメータを学習'''\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 1. メモリサイズの確認\n",
    "        # -----------------------------------------\n",
    "        # 1.1 メモリサイズがミニバッチより小さい間は何もしない\n",
    "        if len(self.memory) < BATCH_SIZE:\n",
    "            return\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 2. ミニバッチの作成\n",
    "        # -----------------------------------------\n",
    "        # 2.1 メモリからミニバッチ分のデータを取り出す\n",
    "        transitions = self.memory.sample(BATCH_SIZE)\n",
    "\n",
    "        # 2.2 各変数をミニバッチに対応する形に変形\n",
    "        # transitionsは1stepごとの(state, action, state_next, reward)が、BATCH_SIZE分格納されている\n",
    "        # つまり、(state, action, state_next, reward)×BATCH_SIZE\n",
    "        # これをミニバッチにしたい。つまり\n",
    "        # (state×BATCH_SIZE, action×BATCH_SIZE, state_next×BATCH_SIZE, reward×BATCH_SIZE)にする\n",
    "        batch = Transition(*zip(*transitions))\n",
    "\n",
    "        # 2.3 各変数の要素をミニバッチに対応する形に変形し、ネットワークで扱えるようVariableにする\n",
    "        # 例えばstateの場合、[torch.FloatTensor of size 1x4]がBATCH_SIZE分並んでいるのですが、\n",
    "        # それを torch.FloatTensor of size BATCH_SIZEx4 に変換します\n",
    "        # 状態、行動、報酬、non_finalの状態のミニバッチのVariableを作成\n",
    "        # catはConcatenates（結合）のことです。\n",
    "        state_batch = torch.cat(batch.state)\n",
    "        action_batch = torch.cat(batch.action)\n",
    "        reward_batch = torch.cat(batch.reward)\n",
    "        non_final_next_states = torch.cat([s for s in batch.next_state\n",
    "                                           if s is not None])\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 3. 教師信号となるQ(s_t, a_t)値を求める\n",
    "        # -----------------------------------------\n",
    "        # 3.1 ネットワークを推論モードに切り替える\n",
    "        self.model.eval()\n",
    "\n",
    "        # 3.2 ネットワークが出力したQ(s_t, a_t)を求める\n",
    "        # self.model(state_batch)は、右左の両方のQ値を出力しており\n",
    "        # [torch.FloatTensor of size BATCH_SIZEx2]になっている。\n",
    "        # ここから実行したアクションa_tに対応するQ値を求めるため、action_batchで行った行動a_tが右か左かのindexを求め\n",
    "        # それに対応するQ値をgatherでひっぱり出す。\n",
    "        state_action_values = self.model(state_batch).gather(1, action_batch)\n",
    "\n",
    "        # 3.3 max{Q(s_t+1, a)}値を求める。ただし次の状態があるかに注意。\n",
    "\n",
    "        # cartpoleがdoneになっておらず、next_stateがあるかをチェックするインデックスマスクを作成\n",
    "        non_final_mask = torch.ByteTensor(tuple(map(lambda s: s is not None,\n",
    "                                                    batch.next_state)))\n",
    "        # まずは全部0にしておく\n",
    "        next_state_values = torch.zeros(BATCH_SIZE)\n",
    "\n",
    "        # 次の状態があるindexの最大Q値を求める\n",
    "        # 出力にアクセスし、max(1)で列方向の最大値の[値、index]を求めます\n",
    "        # そしてそのQ値（index=0）を出力します\n",
    "        # detachでその値を取り出します\n",
    "        next_state_values[non_final_mask] = self.model(\n",
    "            non_final_next_states).max(1)[0].detach()\n",
    "\n",
    "        # 3.4 教師となるQ(s_t, a_t)値を、Q学習の式から求める\n",
    "        expected_state_action_values = reward_batch + GAMMA * next_state_values\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 4. 結合パラメータの更新\n",
    "        # -----------------------------------------\n",
    "        # 4.1 ネットワークを訓練モードに切り替える\n",
    "        self.model.train()\n",
    "\n",
    "        # 4.2 損失関数を計算する（smooth_l1_lossはHuberloss）\n",
    "        # expected_state_action_valuesは\n",
    "        # sizeが[minbatch]になっているので、unsqueezeで[minibatch x 1]へ\n",
    "        loss = F.smooth_l1_loss(state_action_values,\n",
    "                                expected_state_action_values.unsqueeze(1))\n",
    "\n",
    "        # 4.3 結合パラメータを更新する\n",
    "        self.optimizer.zero_grad()  # 勾配をリセット\n",
    "        loss.backward()  # バックプロパゲーションを計算\n",
    "        self.optimizer.step()  # 結合パラメータを更新\n",
    "\n",
    "    def decide_action(self, state, episode):\n",
    "        '''現在の状態に応じて、行動を決定する'''\n",
    "        # ε-greedy法で徐々に最適行動のみを採用する\n",
    "        epsilon = 0.5 * (1 / (episode + 1))\n",
    "\n",
    "        if epsilon <= np.random.uniform(0, 1):\n",
    "            self.model.eval()  # ネットワークを推論モードに切り替える\n",
    "            with torch.no_grad():\n",
    "                action = self.model(state).max(1)[1].view(1, 1)\n",
    "            # ネットワークの出力の最大値のindexを取り出します = max(1)[1]\n",
    "            # .view(1,1)は[torch.LongTensor of size 1]　を size 1x1 に変換します\n",
    "\n",
    "        else:\n",
    "            # 0,1の行動をランダムに返す\n",
    "            action = torch.LongTensor(\n",
    "                [[random.randrange(self.num_actions)]])  # 0,1の行動をランダムに返す\n",
    "            # actionは[torch.LongTensor of size 1x1]の形になります\n",
    "\n",
    "        return action\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CartPoleで動くエージェントクラスです、棒付き台車そのものになります\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        '''課題の状態と行動の数を設定する'''\n",
    "        self.brain = Brain(num_states, num_actions)  # エージェントが行動を決定するための頭脳を生成\n",
    "\n",
    "    def update_q_function(self):\n",
    "        '''Q関数を更新する'''\n",
    "        self.brain.replay()\n",
    "\n",
    "    def get_action(self, state, episode):\n",
    "        '''行動を決定する'''\n",
    "        action = self.brain.decide_action(state, episode)\n",
    "        return action\n",
    "\n",
    "    def memorize(self, state, action, state_next, reward):\n",
    "        '''memoryオブジェクトに、state, action, state_next, rewardの内容を保存する'''\n",
    "        self.brain.memory.push(state, action, state_next, reward)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CartPoleを実行する環境のクラスです\n",
    "\n",
    "\n",
    "class Environment:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.env = gym.make(ENV)  # 実行する課題を設定\n",
    "        num_states = self.env.observation_space.shape[0]  # 課題の状態数4を取得\n",
    "        num_actions = self.env.action_space.n  # CartPoleの行動（右に左に押す）の2を取得\n",
    "        self.agent = Agent(num_states, num_actions)  # 環境内で行動するAgentを生成\n",
    "\n",
    "        \n",
    "    def run(self):\n",
    "        '''実行'''\n",
    "        episode_10_list = np.zeros(10)  # 10試行分の立ち続けたstep数を格納し、平均ステップ数を出力に利用\n",
    "        complete_episodes = 0  # 195step以上連続で立ち続けた試行数\n",
    "        episode_final = False  # 最後の試行フラグ\n",
    "        frames = []  # 最後の試行を動画にするために画像を格納する変数\n",
    "\n",
    "        for episode in range(NUM_EPISODES):  # 最大試行数分繰り返す\n",
    "            observation = self.env.reset()  # 環境の初期化\n",
    "\n",
    "            state = observation  # 観測をそのまま状態sとして使用\n",
    "            state = torch.from_numpy(state).type(\n",
    "                torch.FloatTensor)  # NumPy変数をPyTorchのテンソルに変換\n",
    "            state = torch.unsqueeze(state, 0)  # size 4をsize 1x4に変換\n",
    "\n",
    "            for step in range(MAX_STEPS):  # 1エピソードのループ\n",
    "\n",
    "                if episode_final is True:  # 最終試行ではframesに各時刻の画像を追加していく\n",
    "                    frames.append(self.env.render(mode='rgb_array'))\n",
    "\n",
    "                action = self.agent.get_action(state, episode)  # 行動を求める\n",
    "\n",
    "                # 行動a_tの実行により、s_{t+1}とdoneフラグを求める\n",
    "                # actionから.item()を指定して、中身を取り出す\n",
    "                observation_next, _, done, _ = self.env.step(\n",
    "                    action.item())  # rewardとinfoは使わないので_にする\n",
    "\n",
    "                # 報酬を与える。さらにepisodeの終了評価と、state_nextを設定する\n",
    "                if done:  # ステップ数が200経過するか、一定角度以上傾くとdoneはtrueになる\n",
    "                    state_next = None  # 次の状態はないので、Noneを格納\n",
    "\n",
    "                    # 直近10episodeの立てたstep数リストに追加\n",
    "                    episode_10_list = np.hstack(\n",
    "                        (episode_10_list[1:], step + 1))\n",
    "\n",
    "                    if step < 195:\n",
    "                        reward = torch.FloatTensor(\n",
    "                            [-1.0])  # 途中でこけたら罰則として報酬-1を与える\n",
    "                        complete_episodes = 0  # 連続成功記録をリセット\n",
    "                    else:\n",
    "                        reward = torch.FloatTensor([1.0])  # 立ったまま終了時は報酬1を与える\n",
    "                        complete_episodes = complete_episodes + 1  # 連続記録を更新\n",
    "                else:\n",
    "                    reward = torch.FloatTensor([0.0])  # 普段は報酬0\n",
    "                    state_next = observation_next  # 観測をそのまま状態とする\n",
    "                    state_next = torch.from_numpy(state_next).type(\n",
    "                        torch.FloatTensor)  # numpy変数をPyTorchのテンソルに変換\n",
    "                    state_next = torch.unsqueeze(state_next, 0)  # size 4をsize 1x4に変換\n",
    "\n",
    "                # メモリに経験を追加\n",
    "                self.agent.memorize(state, action, state_next, reward)\n",
    "\n",
    "                # Experience ReplayでQ関数を更新する\n",
    "                self.agent.update_q_function()\n",
    "\n",
    "                # 観測の更新\n",
    "                state = state_next\n",
    "\n",
    "                # 終了時の処理\n",
    "                if done:\n",
    "                    print('%d Episode: Finished after %d steps：10試行の平均step数 = %.1lf' % (\n",
    "                        episode, step + 1, episode_10_list.mean()))\n",
    "                    break\n",
    "\n",
    "            if episode_final is True:\n",
    "                # 動画を保存と描画\n",
    "                display_frames_as_gif(frames)\n",
    "                break\n",
    "\n",
    "            # 10連続で200step経ち続けたら成功\n",
    "            if complete_episodes >= 10:\n",
    "                print('10回連続成功')\n",
    "                episode_final = True  # 次の試行を描画を行う最終試行とする\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "Sequential(\n",
      "  (fc1): Linear(in_features=4, out_features=32, bias=True)\n",
      "  (relu1): ReLU()\n",
      "  (fc2): Linear(in_features=32, out_features=32, bias=True)\n",
      "  (relu2): ReLU()\n",
      "  (fc3): Linear(in_features=32, out_features=2, bias=True)\n",
      ")\n",
      "0 Episode: Finished after 11 steps：10試行の平均step数 = 1.1\n",
      "1 Episode: Finished after 11 steps：10試行の平均step数 = 2.2\n",
      "2 Episode: Finished after 11 steps：10試行の平均step数 = 3.3\n",
      "3 Episode: Finished after 10 steps：10試行の平均step数 = 4.3\n",
      "4 Episode: Finished after 14 steps：10試行の平均step数 = 5.7\n",
      "5 Episode: Finished after 9 steps：10試行の平均step数 = 6.6\n",
      "6 Episode: Finished after 10 steps：10試行の平均step数 = 7.6\n",
      "7 Episode: Finished after 10 steps：10試行の平均step数 = 8.6\n",
      "8 Episode: Finished after 10 steps：10試行の平均step数 = 9.6\n",
      "9 Episode: Finished after 9 steps：10試行の平均step数 = 10.5\n",
      "10 Episode: Finished after 12 steps：10試行の平均step数 = 10.6\n",
      "11 Episode: Finished after 10 steps：10試行の平均step数 = 10.5\n",
      "12 Episode: Finished after 73 steps：10試行の平均step数 = 16.7\n",
      "13 Episode: Finished after 25 steps：10試行の平均step数 = 18.2\n",
      "14 Episode: Finished after 21 steps：10試行の平均step数 = 18.9\n",
      "15 Episode: Finished after 24 steps：10試行の平均step数 = 20.4\n",
      "16 Episode: Finished after 24 steps：10試行の平均step数 = 21.8\n",
      "17 Episode: Finished after 20 steps：10試行の平均step数 = 22.8\n",
      "18 Episode: Finished after 22 steps：10試行の平均step数 = 24.0\n",
      "19 Episode: Finished after 30 steps：10試行の平均step数 = 26.1\n",
      "20 Episode: Finished after 33 steps：10試行の平均step数 = 28.2\n",
      "21 Episode: Finished after 22 steps：10試行の平均step数 = 29.4\n",
      "22 Episode: Finished after 44 steps：10試行の平均step数 = 26.5\n",
      "23 Episode: Finished after 55 steps：10試行の平均step数 = 29.5\n",
      "24 Episode: Finished after 78 steps：10試行の平均step数 = 35.2\n",
      "25 Episode: Finished after 54 steps：10試行の平均step数 = 38.2\n",
      "26 Episode: Finished after 28 steps：10試行の平均step数 = 38.6\n",
      "27 Episode: Finished after 38 steps：10試行の平均step数 = 40.4\n",
      "28 Episode: Finished after 37 steps：10試行の平均step数 = 41.9\n",
      "29 Episode: Finished after 29 steps：10試行の平均step数 = 41.8\n",
      "30 Episode: Finished after 29 steps：10試行の平均step数 = 41.4\n",
      "31 Episode: Finished after 25 steps：10試行の平均step数 = 41.7\n",
      "32 Episode: Finished after 25 steps：10試行の平均step数 = 39.8\n",
      "33 Episode: Finished after 31 steps：10試行の平均step数 = 37.4\n",
      "34 Episode: Finished after 34 steps：10試行の平均step数 = 33.0\n",
      "35 Episode: Finished after 33 steps：10試行の平均step数 = 30.9\n",
      "36 Episode: Finished after 34 steps：10試行の平均step数 = 31.5\n",
      "37 Episode: Finished after 35 steps：10試行の平均step数 = 31.2\n",
      "38 Episode: Finished after 36 steps：10試行の平均step数 = 31.1\n",
      "39 Episode: Finished after 25 steps：10試行の平均step数 = 30.7\n",
      "40 Episode: Finished after 28 steps：10試行の平均step数 = 30.6\n",
      "41 Episode: Finished after 59 steps：10試行の平均step数 = 34.0\n",
      "42 Episode: Finished after 29 steps：10試行の平均step数 = 34.4\n",
      "43 Episode: Finished after 26 steps：10試行の平均step数 = 33.9\n",
      "44 Episode: Finished after 33 steps：10試行の平均step数 = 33.8\n",
      "45 Episode: Finished after 29 steps：10試行の平均step数 = 33.4\n",
      "46 Episode: Finished after 69 steps：10試行の平均step数 = 36.9\n",
      "47 Episode: Finished after 34 steps：10試行の平均step数 = 36.8\n",
      "48 Episode: Finished after 34 steps：10試行の平均step数 = 36.6\n",
      "49 Episode: Finished after 27 steps：10試行の平均step数 = 36.8\n",
      "50 Episode: Finished after 56 steps：10試行の平均step数 = 39.6\n",
      "51 Episode: Finished after 34 steps：10試行の平均step数 = 37.1\n",
      "52 Episode: Finished after 40 steps：10試行の平均step数 = 38.2\n",
      "53 Episode: Finished after 44 steps：10試行の平均step数 = 40.0\n",
      "54 Episode: Finished after 30 steps：10試行の平均step数 = 39.7\n",
      "55 Episode: Finished after 51 steps：10試行の平均step数 = 41.9\n",
      "56 Episode: Finished after 40 steps：10試行の平均step数 = 39.0\n",
      "57 Episode: Finished after 48 steps：10試行の平均step数 = 40.4\n",
      "58 Episode: Finished after 58 steps：10試行の平均step数 = 42.8\n",
      "59 Episode: Finished after 61 steps：10試行の平均step数 = 46.2\n",
      "60 Episode: Finished after 68 steps：10試行の平均step数 = 47.4\n",
      "61 Episode: Finished after 39 steps：10試行の平均step数 = 47.9\n",
      "62 Episode: Finished after 102 steps：10試行の平均step数 = 54.1\n",
      "63 Episode: Finished after 200 steps：10試行の平均step数 = 69.7\n",
      "64 Episode: Finished after 162 steps：10試行の平均step数 = 82.9\n",
      "65 Episode: Finished after 54 steps：10試行の平均step数 = 83.2\n",
      "66 Episode: Finished after 83 steps：10試行の平均step数 = 87.5\n",
      "67 Episode: Finished after 64 steps：10試行の平均step数 = 89.1\n",
      "68 Episode: Finished after 170 steps：10試行の平均step数 = 100.3\n",
      "69 Episode: Finished after 118 steps：10試行の平均step数 = 106.0\n",
      "70 Episode: Finished after 200 steps：10試行の平均step数 = 119.2\n",
      "71 Episode: Finished after 130 steps：10試行の平均step数 = 128.3\n",
      "72 Episode: Finished after 112 steps：10試行の平均step数 = 129.3\n",
      "73 Episode: Finished after 98 steps：10試行の平均step数 = 119.1\n",
      "74 Episode: Finished after 94 steps：10試行の平均step数 = 112.3\n",
      "75 Episode: Finished after 164 steps：10試行の平均step数 = 123.3\n",
      "76 Episode: Finished after 92 steps：10試行の平均step数 = 124.2\n",
      "77 Episode: Finished after 93 steps：10試行の平均step数 = 127.1\n",
      "78 Episode: Finished after 91 steps：10試行の平均step数 = 119.2\n",
      "79 Episode: Finished after 107 steps：10試行の平均step数 = 118.1\n",
      "80 Episode: Finished after 134 steps：10試行の平均step数 = 111.5\n",
      "81 Episode: Finished after 156 steps：10試行の平均step数 = 114.1\n",
      "82 Episode: Finished after 182 steps：10試行の平均step数 = 121.1\n",
      "83 Episode: Finished after 200 steps：10試行の平均step数 = 131.3\n",
      "84 Episode: Finished after 200 steps：10試行の平均step数 = 141.9\n",
      "85 Episode: Finished after 200 steps：10試行の平均step数 = 145.5\n",
      "86 Episode: Finished after 200 steps：10試行の平均step数 = 156.3\n",
      "87 Episode: Finished after 200 steps：10試行の平均step数 = 167.0\n",
      "88 Episode: Finished after 200 steps：10試行の平均step数 = 177.9\n",
      "89 Episode: Finished after 200 steps：10試行の平均step数 = 187.2\n",
      "90 Episode: Finished after 200 steps：10試行の平均step数 = 193.8\n",
      "91 Episode: Finished after 200 steps：10試行の平均step数 = 198.2\n",
      "92 Episode: Finished after 200 steps：10試行の平均step数 = 200.0\n",
      "10回連続成功\n",
      "93 Episode: Finished after 200 steps：10試行の平均step数 = 200.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Define the Animation class */\n",
       "  function Animation(frames, img_id, slider_id, interval, loop_select_id){\n",
       "    this.img_id = img_id;\n",
       "    this.slider_id = slider_id;\n",
       "    this.loop_select_id = loop_select_id;\n",
       "    this.interval = interval;\n",
       "    this.current_frame = 0;\n",
       "    this.direction = 0;\n",
       "    this.timer = null;\n",
       "    this.frames = new Array(frames.length);\n",
       "\n",
       "    for (var i=0; i<frames.length; i++)\n",
       "    {\n",
       "     this.frames[i] = new Image();\n",
       "     this.frames[i].src = frames[i];\n",
       "    }\n",
       "    document.getElementById(this.slider_id).max = this.frames.length - 1;\n",
       "    this.set_frame(this.current_frame);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.get_loop_state = function(){\n",
       "    var button_group = document[this.loop_select_id].state;\n",
       "    for (var i = 0; i < button_group.length; i++) {\n",
       "        var button = button_group[i];\n",
       "        if (button.checked) {\n",
       "            return button.value;\n",
       "        }\n",
       "    }\n",
       "    return undefined;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.set_frame = function(frame){\n",
       "    this.current_frame = frame;\n",
       "    document.getElementById(this.img_id).src = this.frames[this.current_frame].src;\n",
       "    document.getElementById(this.slider_id).value = this.current_frame;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.next_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.min(this.frames.length - 1, this.current_frame + 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.previous_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.max(0, this.current_frame - 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.first_frame = function()\n",
       "  {\n",
       "    this.set_frame(0);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.last_frame = function()\n",
       "  {\n",
       "    this.set_frame(this.frames.length - 1);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.slower = function()\n",
       "  {\n",
       "    this.interval /= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.faster = function()\n",
       "  {\n",
       "    this.interval *= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_forward = function()\n",
       "  {\n",
       "    this.current_frame += 1;\n",
       "    if(this.current_frame < this.frames.length){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.first_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.last_frame();\n",
       "        this.reverse_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.last_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_reverse = function()\n",
       "  {\n",
       "    this.current_frame -= 1;\n",
       "    if(this.current_frame >= 0){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.last_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.first_frame();\n",
       "        this.play_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.first_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.pause_animation = function()\n",
       "  {\n",
       "    this.direction = 0;\n",
       "    if (this.timer){\n",
       "      clearInterval(this.timer);\n",
       "      this.timer = null;\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.play_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = 1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_forward();}, this.interval);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.reverse_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = -1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_reverse();}, this.interval);\n",
       "  }\n",
       "</script>\n",
       "\n",
       "<div class=\"animation\" align=\"center\">\n",
       "    <img id=\"_anim_imgWCNJLZEKWAYRWBLB\">\n",
       "    <br>\n",
       "    <input id=\"_anim_sliderWCNJLZEKWAYRWBLB\" type=\"range\" style=\"width:350px\" name=\"points\" min=\"0\" max=\"1\" step=\"1\" value=\"0\" onchange=\"animWCNJLZEKWAYRWBLB.set_frame(parseInt(this.value));\"></input>\n",
       "    <br>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.slower()\">&#8211;</button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.first_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.previous_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.reverse_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.pause_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.play_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.next_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.last_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animWCNJLZEKWAYRWBLB.faster()\">+</button>\n",
       "  <form action=\"#n\" name=\"_anim_loop_selectWCNJLZEKWAYRWBLB\" class=\"anim_control\">\n",
       "    <input type=\"radio\" name=\"state\" value=\"once\" > Once </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"loop\" checked> Loop </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"reflect\" > Reflect </input>\n",
       "  </form>\n",
       "</div>\n",
       "\n",
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Instantiate the Animation class. */\n",
       "  /* The IDs given should match those used in the template above. */\n",
       "  (function() {\n",
       "    var img_id = \"_anim_imgWCNJLZEKWAYRWBLB\";\n",
       "    var slider_id = \"_anim_sliderWCNJLZEKWAYRWBLB\";\n",
       "    var loop_select_id = \"_anim_loop_selectWCNJLZEKWAYRWBLB\";\n",
       "    var frames = new Array(0);\n",
       "    \n",
       "  frames[0] = \"\"\n",
       "  frames[1] = \"\"\n",
       "  frames[2] = \"\"\n",
       "  frames[3] = \"\"\n",
       "  frames[4] = \"\"\n",
       "  frames[5] = \"\"\n",
       "  frames[6] = \"\"\n",
       "  frames[7] = \"\"\n",
       "  frames[8] = \"\"\n",
       "  frames[9] = \"\"\n",
       "  frames[10] = \"\"\n",
       "  frames[11] = \"\"\n",
       "  frames[12] = \"\"\n",
       "  frames[13] = \"\"\n",
       "  frames[14] = \"\"\n",
       "  frames[15] = \"\"\n",
       "  frames[16] = \"\"\n",
       "  frames[17] = \"\"\n",
       "  frames[18] = \"\"\n",
       "  frames[19] = \"\"\n",
       "  frames[20] = \"\"\n",
       "  frames[21] = \"\"\n",
       "  frames[22] = \"\"\n",
       "  frames[23] = \"\"\n",
       "  frames[24] = \"\"\n",
       "  frames[25] = \"\"\n",
       "  frames[26] = \"\"\n",
       "  frames[27] = \"\"\n",
       "  frames[28] = \"\"\n",
       "  frames[29] = \"\"\n",
       "  frames[30] = \"\"\n",
       "  frames[31] = \"\"\n",
       "  frames[32] = \"\"\n",
       "  frames[33] = \"\"\n",
       "  frames[34] = \"\"\n",
       "  frames[35] = \"\"\n",
       "  frames[36] = \"\"\n",
       "  frames[37] = \"\"\n",
       "  frames[38] = \"\"\n",
       "  frames[39] = \"\"\n",
       "  frames[40] = \"\"\n",
       "  frames[41] = \"\"\n",
       "  frames[42] = \"\"\n",
       "  frames[43] = \"\"\n",
       "  frames[44] = \"\"\n",
       "  frames[45] = \"\"\n",
       "  frames[46] = \"\"\n",
       "  frames[47] = \"\"\n",
       "  frames[48] = \"\"\n",
       "  frames[49] = \"\"\n",
       "  frames[50] = \"\"\n",
       "  frames[51] = \"\"\n",
       "  frames[52] = \"\"\n",
       "  frames[53] = \"\"\n",
       "  frames[54] = \"\"\n",
       "  frames[55] = \"\"\n",
       "  frames[56] = \"\"\n",
       "  frames[57] = \"\"\n",
       "  frames[58] = \"\"\n",
       "  frames[59] = \"\"\n",
       "  frames[60] = \"\"\n",
       "  frames[61] = \"\"\n",
       "  frames[62] = \"\"\n",
       "  frames[63] = \"\"\n",
       "  frames[64] = \"\"\n",
       "  frames[65] = \"\"\n",
       "  frames[66] = \"\"\n",
       "  frames[67] = \"\"\n",
       "  frames[68] = \"\"\n",
       "  frames[69] = \"\"\n",
       "  frames[70] = \"\"\n",
       "  frames[71] = \"\"\n",
       "  frames[72] = \"\"\n",
       "  frames[73] = \"\"\n",
       "  frames[74] = \"\"\n",
       "  frames[75] = \"\"\n",
       "  frames[76] = \"\"\n",
       "  frames[77] = \"\"\n",
       "  frames[78] = \"\"\n",
       "  frames[79] = \"\"\n",
       "  frames[80] = \"\"\n",
       "  frames[81] = \"\"\n",
       "  frames[82] = \"\"\n",
       "  frames[83] = \"\"\n",
       "  frames[84] = \"\"\n",
       "  frames[85] = \"\"\n",
       "  frames[86] = \"\"\n",
       "  frames[87] = \"\"\n",
       "  frames[88] = \"\"\n",
       "  frames[89] = \"\"\n",
       "  frames[90] = \"\"\n",
       "  frames[91] = \"\"\n",
       "  frames[92] = \"\"\n",
       "  frames[93] = \"\"\n",
       "  frames[94] = \"\"\n",
       "  frames[95] = \"\"\n",
       "  frames[96] = \"\"\n",
       "  frames[97] = \"\"\n",
       "  frames[98] = \"\"\n",
       "  frames[99] = \"\"\n",
       "  frames[100] = \"\"\n",
       "  frames[101] = \"\"\n",
       "  frames[102] = \"\"\n",
       "  frames[103] = \"\"\n",
       "  frames[104] = \"\"\n",
       "  frames[105] = \"\"\n",
       "  frames[106] = \"\"\n",
       "  frames[107] = \"\"\n",
       "  frames[108] = \"\"\n",
       "  frames[109] = \"\"\n",
       "  frames[110] = \"\"\n",
       "  frames[111] = \"\"\n",
       "  frames[112] = \"\"\n",
       "  frames[113] = \"\"\n",
       "  frames[114] = \"\"\n",
       "  frames[115] = \"\"\n",
       "  frames[116] = \"\"\n",
       "  frames[117] = \"\"\n",
       "  frames[118] = \"\"\n",
       "  frames[119] = \"\"\n",
       "  frames[120] = \"\"\n",
       "  frames[121] = \"\"\n",
       "  frames[122] = \"\"\n",
       "  frames[123] = \"\"\n",
       "  frames[124] = \"\"\n",
       "  frames[125] = \"\"\n",
       "  frames[126] = \"\"\n",
       "  frames[127] = \"\"\n",
       "  frames[128] = \"\"\n",
       "  frames[129] = \"\"\n",
       "  frames[130] = \"\"\n",
       "  frames[131] = \"\"\n",
       "  frames[132] = \"\"\n",
       "  frames[133] = \"\"\n",
       "  frames[134] = \"\"\n",
       "  frames[135] = \"\"\n",
       "  frames[136] = \"\"\n",
       "  frames[137] = \"\"\n",
       "  frames[138] = \"\"\n",
       "  frames[139] = \"\"\n",
       "  frames[140] = \"\"\n",
       "  frames[141] = \"\"\n",
       "  frames[142] = \"\"\n",
       "  frames[143] = \"\"\n",
       "  frames[144] = \"\"\n",
       "  frames[145] = \"\"\n",
       "  frames[146] = \"\"\n",
       "  frames[147] = \"\"\n",
       "  frames[148] = \"\"\n",
       "  frames[149] = \"\"\n",
       "  frames[150] = \"\"\n",
       "  frames[151] = \"\"\n",
       "  frames[152] = \"\"\n",
       "  frames[153] = \"\"\n",
       "  frames[154] = \"\"\n",
       "  frames[155] = \"\"\n",
       "  frames[156] = \"\"\n",
       "  frames[157] = \"\"\n",
       "  frames[158] = \"\"\n",
       "  frames[159] = \"\"\n",
       "  frames[160] = \"\"\n",
       "  frames[161] = \"\"\n",
       "  frames[162] = \"\"\n",
       "  frames[163] = \"\"\n",
       "  frames[164] = \"\"\n",
       "  frames[165] = \"\"\n",
       "  frames[166] = \"\"\n",
       "  frames[167] = \"\"\n",
       "  frames[168] = \"\"\n",
       "  frames[169] = \"\"\n",
       "  frames[170] = \"\"\n",
       "  frames[171] = \"\"\n",
       "  frames[172] = \"\"\n",
       "  frames[173] = \"\"\n",
       "  frames[174] = \"\"\n",
       "  frames[175] = \"\"\n",
       "  frames[176] = \"\"\n",
       "  frames[177] = \"\"\n",
       "  frames[178] = \"\"\n",
       "  frames[179] = \"\"\n",
       "  frames[180] = \"\"\n",
       "  frames[181] = \"\"\n",
       "  frames[182] = \"\"\n",
       "  frames[183] = \"\"\n",
       "  frames[184] = \"\"\n",
       "  frames[185] = \"\"\n",
       "  frames[186] = \"\"\n",
       "  frames[187] = \"\"\n",
       "  frames[188] = \"\"\n",
       "  frames[189] = \"\"\n",
       "  frames[190] = \"\"\n",
       "  frames[191] = \"\"\n",
       "  frames[192] = \"\"\n",
       "  frames[193] = \"\"\n",
       "  frames[194] = \"\"\n",
       "  frames[195] = \"\"\n",
       "  frames[196] = \"\"\n",
       "  frames[197] = \"\"\n",
       "  frames[198] = \"\"\n",
       "  frames[199] = \"\"\n",
       "\n",
       "\n",
       "    /* set a timeout to make sure all the above elements are created before\n",
       "       the object is initialized. */\n",
       "    setTimeout(function() {\n",
       "        animWCNJLZEKWAYRWBLB = new Animation(frames, img_id, slider_id, 50, loop_select_id);\n",
       "    }, 0);\n",
       "  })()\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# main クラス\n",
    "cartpole_env = Environment()\n",
    "cartpole_env.run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
